{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fd0c0a17-2183-4016-8c15-171a58ed8aa9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\programData\\conda\\env\\ChatGLM2-6B\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n",
      "F:\\programData\\conda\\env\\ChatGLM2-6B\\lib\\site-packages\\numpy\\.libs\\libopenblas.FB5AE2TYXYH2IJRDKGDGQ3XBKLKTF43H.gfortran-win_amd64.dll\n",
      "F:\\programData\\conda\\env\\ChatGLM2-6B\\lib\\site-packages\\numpy\\.libs\\libopenblas64__v0.3.23-246-g3d31191b-gcc_10_3_0.dll\n",
      "  warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[0.2525],\n",
       "        [0.2997]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,1))\n",
    "x = torch.rand(size=(2,4))\n",
    "net(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dfab566c-4a2d-48b3-93a2-76ab25216cf5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('weight', tensor([[ 0.2420,  0.0767, -0.3450, -0.2489,  0.1084,  0.1857, -0.2361,  0.2070]])), ('bias', tensor([0.2874]))])\n"
     ]
    }
   ],
   "source": [
    "# 5.2.1. 参数访问\n",
    "print(net[2].state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0dc5a3a6-fff8-49e2-bcbd-98c3f8b1c554",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.nn.parameter.Parameter'>\n",
      "Parameter containing:\n",
      "tensor([0.2874], requires_grad=True)\n",
      "tensor([0.2874])\n"
     ]
    }
   ],
   "source": [
    "# 5.2.1.1. 目标参数\n",
    "print(type(net[2].bias))\n",
    "print(net[2].bias)\n",
    "print(net[2].bias.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2326db05-24f3-45c2-bf7f-520b9701a58f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[2].weight.grad == None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d0e1c892-5a02-4597-a8ad-2d63368bbba3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))\n",
      "('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))\n"
     ]
    }
   ],
   "source": [
    "# 5.2.1.2. 一次性访问所有参数\n",
    "print(*[(name,param.shape) for name,param in net[0].named_parameters()])\n",
    "print(*[(name,param.shape) for name,param in net.named_parameters()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3b4f23c8-1649-485d-ac2a-e75c085770a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.2874])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()['2.bias'].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "89c3b8b3-ca28-4f2d-b1fc-5544faf8f481",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('0.weight',\n",
       "              tensor([[-0.3605, -0.4577,  0.3131,  0.2453],\n",
       "                      [ 0.2417,  0.2608, -0.2790, -0.4957],\n",
       "                      [-0.0833,  0.2403, -0.1582,  0.1253],\n",
       "                      [-0.0920,  0.2802,  0.2884, -0.4605],\n",
       "                      [ 0.1369, -0.4981, -0.3670,  0.0551],\n",
       "                      [ 0.2145, -0.0649, -0.1354, -0.3796],\n",
       "                      [-0.2478,  0.0305,  0.2289, -0.1056],\n",
       "                      [ 0.4535,  0.1551, -0.1220, -0.1581]])),\n",
       "             ('0.bias',\n",
       "              tensor([-0.4648, -0.3364,  0.1526,  0.0738, -0.1145,  0.0186, -0.2801,  0.3195])),\n",
       "             ('2.weight',\n",
       "              tensor([[ 0.2420,  0.0767, -0.3450, -0.2489,  0.1084,  0.1857, -0.2361,  0.2070]])),\n",
       "             ('2.bias', tensor([0.2874]))])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b00288ba-a1be-4fa8-a6ee-a11b72ac949d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.2981],\n",
       "        [-0.2978]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 5.2.1.3. 从嵌套块收集参数\n",
    "def block1():\n",
    "    return nn.Sequential(nn.Linear(4,8)\n",
    "                         ,nn.ReLU()\n",
    "                         ,nn.Linear(8,4)\n",
    "                         ,nn.ReLU())\n",
    "def block2():\n",
    "    net = nn.Sequential()\n",
    "    for i in range(4):\n",
    "        net.add_module(f'block {i}',block1())\n",
    "    return net\n",
    "\n",
    "rgent = nn.Sequential(block2(),nn.Linear(4,1))\n",
    "rgent(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7aa8e7ba-5864-4078-ad21-b6b62a5e2fbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Sequential(\n",
      "    (block 0): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 1): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 2): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 3): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (1): Linear(in_features=4, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(rgent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "910e539a-1be2-4dfc-b6b6-caadc844bd70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.1380,  0.3023,  0.1194,  0.2451,  0.2825,  0.1974, -0.3113, -0.4589])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgent[0][1][0].bias.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ae54ab5-2510-40a5-b6f3-92fd4ac746f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.2.2. 参数初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6bbefd56-02b7-459e-879a-5b354b33a2e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0.0101, -0.0114, -0.0041,  0.0071]), tensor(0.))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#5.2.2.1. 内置初始化\n",
    "def init_normal(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.normal_(m.weight,mean=0,std=0.01)\n",
    "        nn.init.zeros_(m.bias)\n",
    "net.apply(init_normal)\n",
    "net[0].weight.data[0],net[0].bias.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c9ab0eb6-2c4d-4db1-abef-54eada3e28f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.3558,  0.3792, -0.1647,  0.5043],\n",
      "        [-0.0053,  0.0374, -0.2313, -0.2861],\n",
      "        [-0.0391, -0.6666,  0.1770, -0.1901],\n",
      "        [ 0.6272,  0.6980, -0.4561, -0.5890],\n",
      "        [ 0.3884,  0.6853, -0.5808, -0.4200],\n",
      "        [ 0.1225, -0.5776, -0.6393, -0.3510],\n",
      "        [ 0.0936,  0.0701,  0.3228,  0.6653],\n",
      "        [ 0.4604,  0.3984, -0.6797, -0.1293]])\n",
      "tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])\n"
     ]
    }
   ],
   "source": [
    "def init_xavier(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.xavier_uniform_(m.weight)\n",
    "def init_42(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight,42)\n",
    "\n",
    "net[0].apply(init_xavier)\n",
    "net[2].apply(init_42)\n",
    "print(net[0].weight.data)\n",
    "print(net[2].weight.data)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e0538ff0-564c-4bf3-8434-38acc1078d21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init weight torch.Size([8, 4])\n",
      "Init weight torch.Size([1, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.0000, -8.3835,  0.0000, -0.0000],\n",
       "        [-6.0539,  9.2093, -9.1482, -6.1750],\n",
       "        [-8.6153,  0.0000, -6.3991,  6.5672],\n",
       "        [-0.0000, -0.0000, -8.8401,  0.0000],\n",
       "        [ 0.0000, -0.0000,  0.0000,  0.0000],\n",
       "        [-9.3192,  0.0000,  9.1029,  0.0000],\n",
       "        [-9.9707, -0.0000, -6.1985,  0.0000],\n",
       "        [ 0.0000, -9.8143,  9.9310, -0.0000]], requires_grad=True)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#5.2.2.2. 自定义初始化\n",
    "def my_init(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        print(\"Init\",*[(name,param.shape) for name,param in m.named_parameters()][0])\n",
    "        nn.init.uniform_(m.weight,-10,10)\n",
    "        m.weight.data *= m.weight.data.abs() >= 5\n",
    "net.apply(my_init)\n",
    "net[0].weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "acb36960-0236-490a-992f-6aa6fb6919ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([42.0000, -7.3835,  1.0000,  1.0000])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#直接设置参数\n",
    "net[0].weight.data[:] += 1\n",
    "net[0].weight.data[0,0] = 42\n",
    "net[0].weight.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0b63a792-bb46-49e4-9451-36dee5442e9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.1483],\n",
       "        [-0.1147]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 5.2.3. 参数绑定\n",
    "shared = nn.Linear(8,8)\n",
    "net = nn.Sequential(nn.Linear(4,8)\n",
    "                    ,nn.ReLU()\n",
    "                    ,shared\n",
    "                    ,nn.ReLU()\n",
    "                    ,shared\n",
    "                    ,nn.Linear(8,1))\n",
    "net(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7617bbbc-f4db-416f-a1d7-abdfc8a51293",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([True, True, True, True, True, True, True, True])\n",
      "tensor([True, True, True, True, True, True, True, True])\n"
     ]
    }
   ],
   "source": [
    "print(net[2].weight.data[0] == net[4].weight.data[0])\n",
    "net[2].weight.data[0,0] = 100\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
